import re
import sys
import traceback
import urllib.error
import urllib.parse
import urllib.request
from time import sleep

from proxy_generator import ProxyGenerator

class FreeproxylistsGenerator(ProxyGenerator):
  def __init__(self):
    super().__init__()
    self.pages = [
        'http://www.freeproxylists.net/zh/?page=%d' % i for i in range(1, 38)]
    self._ip_port_pattern = re.compile(
        '<td><script type="text/javascript">IPDecode[^<]+</script>' +
        '(\d*.\d*.\d*.\d*)</td><td align="center">(\d+)</td>')
  
  def GetIps(self, webdata):
    # sleep(10)  # Prevent activating reCaptcha
    webdata = webdata.decode('utf8')
    ip_ports = self._ip_port_pattern.findall(webdata)
    for pair in ip_ports:
      # sleep(1)
      proxy = pair[0] + ':' + pair[1]
      yield proxy
